# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate:
# Message: 获取ecatcorteco.com网站某个品牌下每个车系对应的发动机编号
from MysqlTools import MysqlTools
import os
import requests
import hashlib
import json
from lxml import etree
import sys
# 解决python2编码问题
reload(sys)
sys.setdefaultencoding('utf-8')

class EcatcortecoEngineCode(object):

    def __init__(self, brand_num, **kwargs):
        self.brand_num = brand_num  # 网站的品牌代号
        self.brandCode = kwargs.get("brandCode")
        self.webBrand = kwargs.get("webBrand")
        self.local_db = MysqlTools(db_config=kwargs.get("local_db"), type='dict', auto_commit=100)
        self.start_page = 'http://www.ecatcorteco.com/web/corteco/en_GB/ENG/{}/applicationSearch.xhtml'.format(self.brand_num)
        self.engine_page = 'http://www.ecatcorteco.com/web/na/en_GB/ENG/{}/{}/applicationSearch.xhtml'.format(self.brand_num, '{}')
        # 存储路径代码块
        self.base_path = os.getcwd() + '/../{}_html/'.format(self.webBrand)  # 获取当前路径并拼接出存储html文件的路径
        if not os.path.exists(self.base_path):  # 如果路径不存在,就创建一个
            os.mkdir(self.base_path)
        self.total_num = 0  # 全局计数变量
        self.sub_num = 0  # 全局备用计数变量
        self.is_sleep = kwargs.get("is_sleep", True)  # 是否控制速度的开关变量
        self.insert_table = kwargs.get("insert_table")

        # 网站参数代码块
        self.headers = {
            "Host": "www.ecatcorteco.com",
            "Upgrade-Insecure-Requests": "1",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
            "Referer": "http://www.ecatcorteco.com/web/corteco/en_GB/ENG/applicationSearch.xhtml",
            # "": "",
        }
        self.cookies = {
            "JSESSIONID": "0049B18CD0659CEC37D60463BCB90B33",
            "mode": "SEARCH_APPLICATION",
            "locale": "en_GB",
            "__utma": "44551344.960132916.1539658467.1539658467.1539658467.1",
            "__utmc": "44551344",
            "__utmz": "44551344.1539658467.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
            "__utmt": "1",
            "__utmb": "44551344.12.10.1539658467"
        }

    def get_md5(self, str):
        '''获取md5加密电子指纹'''
        md5 = hashlib.md5()
        md5.update(str)
        return md5.hexdigest()

    def parse_url(self, url, method, post_data=None, headers=None, tofile=True, **kwargs):
        requests.packages.urllib3.disable_warnings()  # 因为把ssl安全验证关了,所以不让代码报不安全的https请求警告
        pid = kwargs.get("local_pid", "")
        charset = kwargs.get("charset", "utf8")  # 暂时没有用到
        allow_redirects = kwargs.get("allow_redirects", False)  # 是否允许重定向, 默认不重定向,获取原始响应
        if method == 'GET':
            data = "GET" + url + pid  # 拼接data
            md5 = self.get_md5(data)  # 获取此次请求电子指纹
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "GET for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:  # 如果路径不存在, 从web端请求
                print "GET for web"
                try:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=30,
                                            allow_redirects=allow_redirects)
                except Exception as e:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=30,
                                            allow_redirects=allow_redirects)

                # 每次请求都获取cookies, 并更新self.cookies(如果获取到cookies的话)
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print "响应头中存在location"
                    self.location = location
                    print location
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None

        elif method == "POST":
            data = "POST" + url + json.dumps(post_data)
            md5 = self.get_md5(data)
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print "POST for local"
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:
                print "POST for web"
                try:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=30,
                                             allow_redirects=allow_redirects)
                except Exception as e:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=30,
                                             allow_redirects=allow_redirects)

                # 获取cookies, 并更新self.cookies
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print "响应头中存在location"
                    self.location = location
                    print location
                    # response, file_path = self.parse_url(url=location, method="GET", headers=headers, local_pid=pid)
                    # print "已重新请求location地址"
                    # return response, file_path
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")

                else:
                    response = None

        return response, file_path

    def _clean(self, v):
        return v.replace(' ', '').replace('\n', '').replace('\t', '')

    def run(self):
        # 访问起始页
        try:
            start_response, start_file_path = self.parse_url(url=self.start_page, method="GET", headers=self.headers, allow_redirects=True)
        except Exception as e:
            start_response, start_file_path = self.parse_url(url=self.start_page, method="GET", headers=self.headers, allow_redirects=True)
        try:
            start_html = etree.HTML(start_response)
        except Exception as e:
            print "获取起始页失败, {}".format(start_file_path)
            raise e
        # 获取车系代号列表
        model_option_list = start_html.xpath('//select[@class="series"]/option[position()>2]')
        if len(model_option_list) == 0:
            print "没有获取到车系下拉列表"
            exit()
        # 遍历拼接engine_page并访问
        for model_option in model_option_list:
            model = model_option.xpath('./text()')
            model = str(model[0]) if len(model)>0 else ""
            model_num = model_option.xpath('./@value')
            model_num = model_num[0] if len(model_num)>0 else ""
            if model=="" or model_num=="":
                print "获取车系及车系代号错误, {},{}".format(model, model_num)
                continue
            # 拼接engine_page
            engine_page = self.engine_page.format(model_num)
            try:
                engine_response, engine_file_path = self.parse_url(url=engine_page, method="GET", headers=self.headers, allow_redirects=True)
            except:
                engine_response, engine_file_path = self.parse_url(url=engine_page, method="GET", headers=self.headers, allow_redirects=True)
            try:
                engine_html = etree.HTML(engine_response)
            except Exception as e:
                print "获取发动机页失败, {}".format(engine_file_path)
                raise e
            # 获取所需数据
            # 获取tr_list
            tr_list = engine_html.xpath('//div[@class="valueTable"]/table[1]/tbody/tr')
            if len(tr_list) == 0:
                print "获取engine数据列表错误, {}, {}".format(model, model_num)
                # values = engine_html.xpath('//td[@class="parameter-value"]/div/text()')
                # Vehicle = ''
                # ConstructionYear = self._clean(values[0])
                # EngineOutput = self._clean(values[1])
                # FuelType = self._clean(values[2])
                # Displacement = self._clean(values[3])
                # EngineCode = self._clean(values[4])
                continue
            for tr in tr_list:
                Vehicle = tr.xpath('./td[1]/a/text()')
                ConstructionYear = tr.xpath('./td[2]/div/text()')
                EngineOutput = tr.xpath('./td[3]/div/text()')
                FuelType = tr.xpath('./td[4]/div/text()')
                Displacement = tr.xpath('./td[5]/div/text()')
                EngineCode = tr.xpath('./td[6]/div/text()')

                Vehicle = Vehicle[0] if len(Vehicle)>0 else ""
                ConstructionYear = ConstructionYear[0] if len(ConstructionYear)>0 else ""
                EngineOutput = EngineOutput[0] if len(EngineOutput)>0 else ""
                FuelType = FuelType[0] if len(FuelType)>0 else ""
                Displacement = Displacement[0] if len(Displacement)>0 else ""
                EngineCode = EngineCode[0] if len(EngineCode)>0 else ""

                Vehicle = self._clean(Vehicle)
                ConstructionYear = self._clean(ConstructionYear)
                EngineOutput = self._clean(EngineOutput)
                FuelType = self._clean(FuelType)
                Displacement = self._clean(Displacement)
                EngineCode = self._clean(EngineCode)

                # print Vehicle
                # print ConstructionYear
                # print EngineOutput
                # print FuelType
                # print Displacement
                # print EngineCode
                # exit()
                EngineCode = EngineCode.split('/')
                for i in EngineCode:
                    _dict = dict(
                        model=model,
                        brandCode=self.brandCode,
                        Vehicle=Vehicle,
                        ConstructionYear=ConstructionYear,
                        EngineOutput=EngineOutput,
                        FuelType=FuelType,
                        Displacement=Displacement,
                        EngineCode=i
                    )
                    print _dict
                    insert_sql = self.local_db.get_insert_sql(self.insert_table, _dict)
                    self.local_db.execute_sql(insert_sql)
        self.local_db.commit_sql()
        print "完毕..."


if __name__ == '__main__':
    local_db = dict(
        host="127.0.0.1",
        port=3306,
        user="root",
        passwd="123456",
        db="other_spider",
        charset="utf8",
    )
    webBrand = 'ecatcorteco'  # 网站品牌
    brandCode = 'ferrari'  # 需要的品牌
    brand_num = '700'  # 网站品牌代号, 需要自行去网站看源代码
    insert_table = 'ecatcorteco_engineCode_volvos'
    tools = EcatcortecoEngineCode(brandCode=brandCode, brand_num=brand_num, local_db=local_db, webBrand=webBrand, insert_table=insert_table)
    tools.run()